1
|
|
|
/** |
2
|
|
|
* Wappalyzer v4 |
3
|
|
|
* |
4
|
|
|
* Created by Elbert Alias <[email protected]> |
5
|
|
|
* |
6
|
|
|
* License: GPLv3 http://www.gnu.org/licenses/gpl-3.0.txt |
7
|
|
|
*/ |
8
|
|
|
|
9
|
|
|
var wappalyzer = (function() { |
10
|
|
|
//'use strict'; |
11
|
|
|
|
12
|
|
|
/** |
13
|
|
|
* Application class |
14
|
|
|
*/ |
15
|
|
|
var Application = function(app, detected) { |
16
|
|
|
this.app = app; |
17
|
|
|
this.confidence = {}; |
18
|
|
|
this.confidenceTotal = 0; |
19
|
|
|
this.detected = Boolean(detected); |
20
|
|
|
this.excludes = []; |
21
|
|
|
this.version = ''; |
22
|
|
|
this.versions = []; |
23
|
|
|
}; |
24
|
|
|
|
25
|
|
|
Application.prototype = { |
26
|
|
|
/** |
27
|
|
|
* Calculate confidence total |
28
|
|
|
*/ |
29
|
|
|
getConfidence: function() { |
30
|
|
|
var total = 0, id; |
31
|
|
|
|
32
|
|
|
for ( id in this.confidence ) { |
33
|
|
|
total += this.confidence[id]; |
34
|
|
|
} |
35
|
|
|
|
36
|
|
|
return this.confidenceTotal = Math.min(total, 100); |
37
|
|
|
}, |
38
|
|
|
|
39
|
|
|
/** |
40
|
|
|
* Resolve version number (find the longest version number that contains all shorter detected version numbers) |
41
|
|
|
*/ |
42
|
|
|
getVersion: function() { |
43
|
|
|
var i, resolved; |
44
|
|
|
|
45
|
|
|
if ( !this.versions.length ) { |
46
|
|
|
return; |
47
|
|
|
} |
48
|
|
|
|
49
|
|
|
this.versions.sort(function(a, b) { |
50
|
|
|
return a.length - b.length; |
51
|
|
|
}); |
52
|
|
|
|
53
|
|
|
resolved = this.versions[0]; |
54
|
|
|
|
55
|
|
|
for ( i = 1; i < this.versions.length; i++ ) { |
56
|
|
|
if ( this.versions[i].indexOf(resolved) === -1 ) { |
57
|
|
|
break; |
58
|
|
|
} |
59
|
|
|
|
60
|
|
|
resolved = this.versions[i]; |
61
|
|
|
} |
62
|
|
|
|
63
|
|
|
return this.version = resolved; |
64
|
|
|
}, |
65
|
|
|
|
66
|
|
|
setDetected: function(pattern, type, value, key) { |
67
|
|
|
this.detected = true; |
68
|
|
|
|
69
|
|
|
// Set confidence level |
70
|
|
|
this.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100; |
71
|
|
|
|
72
|
|
|
// Detect version number |
73
|
|
|
if ( pattern.version ) { |
74
|
|
|
var |
75
|
|
|
version = pattern.version, |
76
|
|
|
matches = pattern.regex.exec(value); |
77
|
|
|
|
78
|
|
|
if ( matches ) { |
79
|
|
|
matches.forEach(function(match, i) { |
80
|
|
|
// Parse ternary operator |
81
|
|
|
var ternary = new RegExp('\\\\' + i + '\\?([^:]+):(.*)$').exec(version); |
82
|
|
|
|
83
|
|
|
if ( ternary && ternary.length === 3 ) { |
84
|
|
|
w.log({ match: match, i: i, ternary: ternary }); |
85
|
|
|
|
86
|
|
|
version = version.replace(ternary[0], match ? ternary[1] : ternary[2]); |
87
|
|
|
|
88
|
|
|
w.log({ version: version }); |
89
|
|
|
} |
90
|
|
|
|
91
|
|
|
// Replace back references |
92
|
|
|
version = version.replace(new RegExp('\\\\' + i, 'g'), match ? match : ''); |
93
|
|
|
}); |
94
|
|
|
|
95
|
|
|
if ( version && this.versions.indexOf(version) < 0 ) { |
96
|
|
|
this.versions.push(version); |
97
|
|
|
} |
98
|
|
|
|
99
|
|
|
this.getVersion(); |
100
|
|
|
} |
101
|
|
|
} |
102
|
|
|
} |
103
|
|
|
}; |
104
|
|
|
|
105
|
|
|
/** |
106
|
|
|
* Call driver functions |
107
|
|
|
*/ |
108
|
|
|
var driver = function(func, args) { |
109
|
|
|
if ( typeof w.driver[func] !== 'function' ) { |
110
|
|
|
w.log('not implemented: w.driver.' + func, 'warn'); |
111
|
|
|
|
112
|
|
|
return; |
113
|
|
|
} |
114
|
|
|
|
115
|
|
|
if ( func !== 'log' ) { |
116
|
|
|
w.log('w.driver.' + func); |
117
|
|
|
} |
118
|
|
|
|
119
|
|
|
return w.driver[func](args); |
120
|
|
|
}; |
121
|
|
|
|
122
|
|
|
/** |
123
|
|
|
* Parse apps.json patterns |
124
|
|
|
*/ |
125
|
|
|
var parsePatterns = function(patterns) { |
126
|
|
|
var |
127
|
|
|
key, |
128
|
|
|
parsed = {}; |
129
|
|
|
|
130
|
|
|
// Convert array to object containing array |
131
|
|
|
if ( patterns instanceof Array ) { |
132
|
|
|
patterns = { main: patterns } |
133
|
|
|
} |
134
|
|
|
|
135
|
|
|
// Convert string to object containing array containing string |
136
|
|
|
if ( typeof patterns === 'string' ) { |
137
|
|
|
patterns = { main: [ patterns ] }; |
138
|
|
|
} |
139
|
|
|
|
140
|
|
|
for ( key in patterns ) { |
141
|
|
|
parsed[key] = []; |
142
|
|
|
|
143
|
|
|
// Convert string to array containing string |
144
|
|
|
if ( typeof patterns[key] === 'string' ) { |
145
|
|
|
patterns[key] = [ patterns[key] ]; |
146
|
|
|
} |
147
|
|
|
|
148
|
|
|
patterns[key].forEach(function(pattern) { |
149
|
|
|
var attrs = {}; |
150
|
|
|
|
151
|
|
|
pattern.split('\\;').forEach(function(attr, i) { |
152
|
|
|
if ( i ) { |
153
|
|
|
// Key value pairs |
154
|
|
|
attr = attr.split(':'); |
155
|
|
|
|
156
|
|
|
if ( attr.length > 1 ) { |
157
|
|
|
attrs[attr.shift()] = attr.join(':'); |
158
|
|
|
} |
159
|
|
|
} else { |
160
|
|
|
attrs.string = attr; |
161
|
|
|
|
162
|
|
|
try { |
163
|
|
|
attrs.regex = new RegExp(attr.replace('/', '\/'), 'i'); // Escape slashes in regular expression |
164
|
|
|
} catch (e) { |
165
|
|
|
attrs.regex = new RegExp(); |
166
|
|
|
|
167
|
|
|
w.log(e + ': ' + attr, 'error'); |
168
|
|
|
} |
169
|
|
|
} |
170
|
|
|
}); |
171
|
|
|
|
172
|
|
|
parsed[key].push(attrs); |
173
|
|
|
}); |
174
|
|
|
} |
175
|
|
|
|
176
|
|
|
// Convert back to array if the original pattern list was an array (or string) |
177
|
|
|
if ( parsed.hasOwnProperty('main') ) { |
178
|
|
|
parsed = parsed.main; |
179
|
|
|
} |
180
|
|
|
|
181
|
|
|
return parsed; |
182
|
|
|
}; |
183
|
|
|
|
184
|
|
|
/** |
185
|
|
|
* Main script |
186
|
|
|
*/ |
187
|
|
|
var w = { |
188
|
|
|
apps: {}, |
189
|
|
|
cats: null, |
190
|
|
|
ping: { hostnames: {} }, |
191
|
|
|
adCache: [], |
192
|
|
|
detected: {}, |
193
|
|
|
|
194
|
|
|
config: { |
195
|
|
|
websiteURL: 'https://wappalyzer.com/', |
196
|
|
|
twitterURL: 'https://twitter.com/Wappalyzer', |
197
|
|
|
githubURL: 'https://github.com/AliasIO/Wappalyzer', |
198
|
|
|
}, |
199
|
|
|
|
200
|
|
|
/** |
201
|
|
|
* Log messages to console |
202
|
|
|
*/ |
203
|
|
|
log: function(message, type) { |
204
|
|
|
if ( type === undefined ) { |
205
|
|
|
type = 'debug'; |
206
|
|
|
} |
207
|
|
|
|
208
|
|
|
if ( typeof message === 'object' ) { |
209
|
|
|
message = JSON.stringify(message); |
210
|
|
|
} |
211
|
|
|
|
212
|
|
|
driver('log', { message: message, type: type }); |
213
|
|
|
}, |
214
|
|
|
|
215
|
|
|
/** |
216
|
|
|
* Initialize |
217
|
|
|
*/ |
218
|
|
|
init: function() { |
219
|
|
|
w.log('w.init'); |
220
|
|
|
|
221
|
|
|
// Checks |
222
|
|
|
if ( w.driver === undefined ) { |
223
|
|
|
w.log('no driver, exiting'); |
224
|
|
|
|
225
|
|
|
return; |
226
|
|
|
} |
227
|
|
|
|
228
|
|
|
// Initialize driver |
229
|
|
|
driver('init'); |
230
|
|
|
}, |
231
|
|
|
|
232
|
|
|
/** |
233
|
|
|
* Analyze the request |
234
|
|
|
*/ |
235
|
|
|
analyze: function(hostname, url, data) { |
236
|
|
|
var |
237
|
|
|
confirmMatch, |
238
|
|
|
apps = {}, |
239
|
|
|
excludes = [], |
240
|
|
|
checkImplies = true, |
241
|
|
|
patterns = []; |
|
|
|
|
242
|
|
|
|
243
|
|
|
w.log('w.analyze'); |
244
|
|
|
|
245
|
|
|
if ( w.apps === undefined || w.categories === undefined ) { |
246
|
|
|
w.log('apps.json not loaded, check for syntax errors'); |
247
|
|
|
|
248
|
|
|
return; |
249
|
|
|
} |
250
|
|
|
|
251
|
|
|
// Remove hash from URL |
252
|
|
|
data.url = url = url.split('#')[0]; |
253
|
|
|
|
254
|
|
|
if ( typeof data.html !== 'string' ) { |
255
|
|
|
data.html = ''; |
256
|
|
|
} |
257
|
|
|
|
258
|
|
|
if ( w.detected[url] === undefined ) { |
259
|
|
|
w.detected[url] = {}; |
260
|
|
|
} |
261
|
|
|
|
262
|
|
|
for ( app in w.apps ) { |
263
|
|
|
apps[app] = w.detected[url] && w.detected[url][app] ? w.detected[url][app] : new Application(app); |
264
|
|
|
|
265
|
|
|
for ( type in w.apps[app] ) { |
|
|
|
|
266
|
|
|
patterns = parsePatterns(w.apps[app][type]); |
267
|
|
|
|
268
|
|
|
if ( !patterns ) { |
269
|
|
|
continue; |
270
|
|
|
} |
271
|
|
|
|
272
|
|
|
confirmMatch = function(pattern, value, key) { |
273
|
|
|
apps[app].setDetected(pattern, type, value, key); |
274
|
|
|
} |
275
|
|
|
|
276
|
|
|
switch ( type ) { |
277
|
|
|
case 'url': |
278
|
|
|
if ( url ) { |
279
|
|
|
w.analyzeUrl(patterns, url, confirmMatch); |
280
|
|
|
} |
281
|
|
|
|
282
|
|
|
break; |
283
|
|
|
case 'html': |
284
|
|
|
if ( data.html ) { |
285
|
|
|
w.analyzeHtml(patterns, data.html, confirmMatch); |
286
|
|
|
} |
287
|
|
|
|
288
|
|
|
break; |
289
|
|
|
case 'script': |
290
|
|
|
if ( data.html ) { |
291
|
|
|
w.analyzeScript(patterns, data.html, confirmMatch); |
292
|
|
|
} |
293
|
|
|
|
294
|
|
|
break; |
295
|
|
|
case 'meta': |
296
|
|
|
if ( data.html ) { |
297
|
|
|
w.analyzeMeta(patterns, data.html, confirmMatch); |
298
|
|
|
} |
299
|
|
|
|
300
|
|
|
break; |
301
|
|
|
case 'headers': |
302
|
|
|
if ( data.hasOwnProperty('headers') && data.headers ) { |
303
|
|
|
w.analyzeHeaders(patterns, data.headers, confirmMatch); |
304
|
|
|
} |
305
|
|
|
|
306
|
|
|
break; |
307
|
|
|
case 'env': |
308
|
|
|
if ( data.hasOwnProperty('env') && data.env ) { |
309
|
|
|
w.analyzeEnv(patterns, data.env, confirmMatch); |
310
|
|
|
} |
311
|
|
|
|
312
|
|
|
break; |
313
|
|
|
default: |
314
|
|
|
} |
315
|
|
|
} |
316
|
|
|
} |
317
|
|
|
|
318
|
|
|
for ( app in apps ) { |
319
|
|
|
if ( !apps[app].detected ) { |
320
|
|
|
delete apps[app]; |
321
|
|
|
} |
322
|
|
|
} |
323
|
|
|
|
324
|
|
|
// Exclude app in detected apps only |
325
|
|
|
for ( app in apps ) { |
326
|
|
|
if (w.apps[app].excludes ) { |
327
|
|
|
if ( typeof w.apps[app].excludes === 'string' ) { |
328
|
|
|
w.apps[app].excludes = [ w.apps[app].excludes ]; |
329
|
|
|
} |
330
|
|
|
|
331
|
|
|
w.apps[app].excludes.forEach(function(excluded) { |
332
|
|
|
excludes.push(excluded); |
333
|
|
|
}); |
334
|
|
|
} |
335
|
|
|
} |
336
|
|
|
|
337
|
|
|
// Remove excluded applications |
338
|
|
|
for ( app in apps ) { |
339
|
|
|
if ( excludes.indexOf(app) !== -1 ) { |
340
|
|
|
delete apps[app]; |
341
|
|
|
} |
342
|
|
|
} |
343
|
|
|
|
344
|
|
|
// Implied applications |
345
|
|
|
// Run several passes as implied apps may imply other apps |
346
|
|
|
while ( checkImplies ) { |
347
|
|
|
checkImplies = false; |
348
|
|
|
|
349
|
|
|
for ( app in apps ) { |
350
|
|
|
confidence = apps[app].confidence; |
|
|
|
|
351
|
|
|
|
352
|
|
|
if ( w.apps[app] && w.apps[app].implies ) { |
353
|
|
|
// Cast strings to an array |
354
|
|
|
if ( typeof w.apps[app].implies === 'string' ) { |
355
|
|
|
w.apps[app].implies = [ w.apps[app].implies ]; |
356
|
|
|
} |
357
|
|
|
|
358
|
|
|
w.apps[app].implies.forEach(function(implied) { |
359
|
|
|
implied = parsePatterns(implied)[0]; |
360
|
|
|
|
361
|
|
|
if ( !w.apps[implied.string] ) { |
362
|
|
|
w.log('Implied application ' + implied.string + ' does not exist', 'warn'); |
363
|
|
|
|
364
|
|
|
return; |
365
|
|
|
} |
366
|
|
|
|
367
|
|
|
if ( !apps.hasOwnProperty(implied.string) ) { |
368
|
|
|
apps[implied.string] = w.detected[url] && w.detected[url][implied.string] ? w.detected[url][implied.string] : new Application(implied.string, true); |
369
|
|
|
|
370
|
|
|
checkImplies = true; |
371
|
|
|
} |
372
|
|
|
|
373
|
|
|
// Apply app confidence to implied app |
374
|
|
|
for ( id in confidence ) { |
|
|
|
|
375
|
|
|
apps[implied.string].confidence[id + ' implied by ' + app] = confidence[id] * ( implied.confidence ? implied.confidence / 100 : 1 ); |
376
|
|
|
} |
377
|
|
|
}); |
378
|
|
|
} |
379
|
|
|
} |
380
|
|
|
} |
381
|
|
|
|
382
|
|
|
w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url); |
383
|
|
|
|
384
|
|
|
// Keep history of detected apps |
385
|
|
|
for ( app in apps ) { |
386
|
|
|
confidence = apps[app].confidence; |
|
|
|
|
387
|
|
|
version = apps[app].version; |
|
|
|
|
388
|
|
|
|
389
|
|
|
// Per URL |
390
|
|
|
w.detected[url][app] = apps[app]; |
391
|
|
|
|
392
|
|
|
for ( id in confidence ) { |
393
|
|
|
w.detected[url][app].confidence[id] = confidence[id]; |
394
|
|
|
} |
395
|
|
|
|
396
|
|
|
if ( w.detected[url][app].getConfidence() >= 100 ) { |
397
|
|
|
// Per hostname |
398
|
|
|
if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/.test(url) ) { |
399
|
|
|
if ( !w.ping.hostnames.hasOwnProperty(hostname) ) { |
400
|
|
|
w.ping.hostnames[hostname] = { |
401
|
|
|
applications: {}, |
402
|
|
|
meta: {} |
403
|
|
|
}; |
404
|
|
|
} |
405
|
|
|
|
406
|
|
|
if ( !w.ping.hostnames[hostname].applications.hasOwnProperty(app) ) { |
407
|
|
|
w.ping.hostnames[hostname].applications[app] = { |
408
|
|
|
hits: 0 |
409
|
|
|
}; |
410
|
|
|
} |
411
|
|
|
|
412
|
|
|
w.ping.hostnames[hostname].applications[app].hits ++; |
413
|
|
|
|
414
|
|
|
if ( version ) { |
415
|
|
|
w.ping.hostnames[hostname].applications[app].version = version; |
416
|
|
|
} |
417
|
|
|
} else { |
418
|
|
|
w.log('Ignoring hostname "' + hostname + '"'); |
419
|
|
|
} |
420
|
|
|
} |
421
|
|
|
} |
422
|
|
|
|
423
|
|
|
// Additional information |
424
|
|
|
if ( w.ping.hostnames.hasOwnProperty(hostname) ) { |
425
|
|
|
if ( typeof data.html === 'string' && data.html ) { |
426
|
|
|
match = data.html.match(/<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i); |
|
|
|
|
427
|
|
|
|
428
|
|
|
if ( match && match.length ) { |
429
|
|
|
w.ping.hostnames[hostname].meta['language'] = match[1]; |
430
|
|
|
} |
431
|
|
|
} |
432
|
|
|
} |
433
|
|
|
|
434
|
|
|
if ( Object.keys(w.ping.hostnames).length >= 50 || w.adCache.length >= 50 ) { |
435
|
|
|
driver('ping'); |
436
|
|
|
} |
437
|
|
|
|
438
|
|
|
driver('displayApps'); |
439
|
|
|
}, |
440
|
|
|
|
441
|
|
|
/** |
442
|
|
|
* Analyze URL |
443
|
|
|
*/ |
444
|
|
|
analyzeUrl: function(patterns, url, confirmMatch) { |
445
|
|
|
patterns.forEach(function(pattern) { |
446
|
|
|
if ( pattern.regex.test(url) ) { |
447
|
|
|
confirmMatch(pattern, url); |
448
|
|
|
} |
449
|
|
|
}); |
450
|
|
|
}, |
451
|
|
|
|
452
|
|
|
/** |
453
|
|
|
* Analyze HTML |
454
|
|
|
*/ |
455
|
|
|
analyzeHtml: function(patterns, html, confirmMatch) { |
456
|
|
|
patterns.forEach(function(pattern) { |
457
|
|
|
if ( pattern.regex.test(html) ) { |
458
|
|
|
confirmMatch(pattern, html); |
459
|
|
|
} |
460
|
|
|
}); |
461
|
|
|
}, |
462
|
|
|
|
463
|
|
|
/** |
464
|
|
|
* Analyze script tag |
465
|
|
|
*/ |
466
|
|
|
analyzeScript: function(patterns, html, confirmMatch) { |
467
|
|
|
var regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig'); |
468
|
|
|
|
469
|
|
|
patterns.forEach(function(pattern) { |
470
|
|
|
var match; |
471
|
|
|
|
472
|
|
|
while ( match = regex.exec(html) ) { |
473
|
|
|
if ( pattern.regex.test(match[2]) ) { |
474
|
|
|
confirmMatch(pattern, match[2]); |
475
|
|
|
} |
476
|
|
|
} |
477
|
|
|
}); |
478
|
|
|
}, |
479
|
|
|
|
480
|
|
|
/** |
481
|
|
|
* Analyze meta tag |
482
|
|
|
*/ |
483
|
|
|
analyzeMeta: function(patterns, html, confirmMatch) { |
484
|
|
|
var |
485
|
|
|
meta, |
486
|
|
|
regex = /<meta[^>]+>/ig; |
487
|
|
|
|
488
|
|
|
while ( match = regex.exec(html) ) { |
|
|
|
|
489
|
|
|
for ( meta in patterns ) { |
490
|
|
|
if ( new RegExp('(name|property)=["\']' + meta + '["\']', 'i').test(match) ) { |
491
|
|
|
content = match.toString().match(/content=("|')([^"']+)("|')/i); |
|
|
|
|
492
|
|
|
|
493
|
|
|
patterns[meta].forEach(function(pattern) { |
494
|
|
|
if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { |
495
|
|
|
confirmMatch(pattern, content[2], meta); |
496
|
|
|
} |
497
|
|
|
}); |
498
|
|
|
} |
499
|
|
|
} |
500
|
|
|
} |
501
|
|
|
}, |
502
|
|
|
|
503
|
|
|
/** |
504
|
|
|
* analyze response headers |
505
|
|
|
*/ |
506
|
|
|
analyzeHeaders: function(patterns, headers, confirmMatch) { |
507
|
|
|
for ( header in patterns ) { |
508
|
|
|
patterns[header].forEach(function(pattern) { |
509
|
|
|
header = header.toLowerCase(); |
|
|
|
|
510
|
|
|
|
511
|
|
|
if ( headers.hasOwnProperty(header) && pattern.regex.test(headers[header]) ) { |
512
|
|
|
confirmMatch(pattern, headers[header], header); |
513
|
|
|
} |
514
|
|
|
}); |
515
|
|
|
} |
516
|
|
|
}, |
517
|
|
|
|
518
|
|
|
/** |
519
|
|
|
* Analyze environment variables |
520
|
|
|
*/ |
521
|
|
|
analyzeEnv: function(patterns, envs, confirmMatch) { |
522
|
|
|
var env; |
523
|
|
|
|
524
|
|
|
patterns.forEach(function(pattern) { |
525
|
|
|
for ( env in envs ) { |
526
|
|
|
if ( pattern.regex.test(envs[env]) ) { |
527
|
|
|
confirmMatch(pattern, envs[env]); |
528
|
|
|
} |
529
|
|
|
} |
530
|
|
|
}); |
531
|
|
|
} |
532
|
|
|
}; |
533
|
|
|
|
534
|
|
|
return w; |
535
|
|
|
})(); |
536
|
|
|
|
537
|
|
|
// CommonJS package |
538
|
|
|
// See http://wiki.commonjs.org/wiki/CommonJS |
539
|
|
|
if ( typeof exports === 'object' ) { |
540
|
|
|
exports.wappalyzer = wappalyzer; |
541
|
|
|
} |
542
|
|
|
|